package com.zhenjiashi.sideline.cralwer;

import com.google.common.collect.Lists;
import com.zhenjiashi.sideline.entity.Jobs;
import com.zhenjiashi.sideline.service.IJobsService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.pipeline.Pipeline;

import java.util.List;
import java.util.Objects;

@Slf4j
@Component
public class DataPipeLine implements Pipeline {

    @Autowired
    private StringRedisTemplate stringTemplate;

    @Autowired
    private IJobsService jobsService;

    @Override
    public void process(ResultItems resultItems, Task task) {
        List<Jobs> jobs = resultItems.get("jobs");
        saveAll(jobs);
    }

    public void saveAll(List<Jobs> jobs) {
        try {
            if (!CollectionUtils.isEmpty(jobs)) {
                List<Jobs> saveList = Lists.newLinkedList();

                String platform = jobs.get(0).getPlatform();
                String url = jobs.get(0).getUrl();

                String lastValue = stringTemplate.opsForValue().getAndSet(platform, url);
                for (Jobs job : jobs) {
                    if (Objects.nonNull(job)) {
                        if (StringUtils.isNotBlank(lastValue) && lastValue.equalsIgnoreCase(job.getUrl())) {
                            break;
                        }
                        saveList.add(job);
                    }
                }

                if (!CollectionUtils.isEmpty(saveList)) {
                    jobsService.saveBatch(saveList);
                }
            }
        } catch (Exception e) {
            log.error("DataPipeLine error", e);
        }
    }
}
